// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved.
 * Author: Huawei OS Kernel Lab
 * Create: Thu Aug 15 16:14:23 2023
 */

#include <linux/device.h>
#include <linux/idr.h>
#include <linux/tty.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/radix-tree.h>
#include <linux/async.h>
#include <linux/slab.h>
#include <linux/console.h>
#include <linux/vmalloc.h>
#include <linux/hrtimer.h>
#include <linux/memblock.h>
#include <asm/sections.h>
#include <linux/stackprotector.h>
#include <asm/kasan.h>
#include <asm/setup.h>
#include <asm/smp.h>
#include <asm/cputype.h>
#include <asm/uaccess.h>
#include <asm/bootparam.h>
#include <linux/buffer_head.h>
#include <linux/of.h>
#include <linux/of_fdt.h>
#include <linux/of_irq.h>
#include <linux/of_iommu.h>
#include <linux/clk-provider.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/random.h>
#include <linux/cpu.h>
#include <linux/key.h>
#include <linux/fs_struct.h>
#include <linux/kmemleak.h>
#include <linux/cpuhotplug.h>
#include <linux/sched/clock.h>

#include "../drivers/of/of_private.h"
#include "../mm/internal.h"	/* for page_writeback_init()
				   TODO: replace by pagecache_init in mm/filemap.c */

/*
 * Debug helper: via this flag we know that we are in 'early bootup code'
 * where only the boot processor is running with IRQ disabled.  This means
 * two things - IRQ must not be enabled before the flag is cleared and some
 * operations which are not allowed with IRQ disabled are allowed while the
 * flag is set.
 */
bool early_boot_irqs_disabled;

enum system_states system_state = SYSTEM_BOOTING;
EXPORT_SYMBOL(system_state);

/* from "arch/arm/mm/mmu.c" */
/*
 * Empty_zero_page is a special page that is used for zero-initialized data
 * and COW.
 */
struct page *empty_zero_page;
EXPORT_SYMBOL(empty_zero_page);

#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
bool rodata_enabled __ro_after_init = true;
static int __init set_debug_rodata(char *str)
{
	return strtobool(str, &rodata_enabled);
}
__setup("rodata=", set_debug_rodata);
#endif

bool initcall_debug;

int __init_or_module do_one_initcall(initcall_t fn)
{
	return fn();
}

extern initcall_t __initcall_start[];
extern initcall_t __initcall0_start[];
extern initcall_t __initcall1_start[];
extern initcall_t __initcall2_start[];
extern initcall_t __initcall3_start[];
extern initcall_t __initcall4_start[];
extern initcall_t __initcall5_start[];
extern initcall_t __initcall6_start[];
extern initcall_t __initcall7_start[];
extern initcall_t __initcall_end[];

static initcall_t *initcall_levels[] __initdata = {
	__initcall0_start,
	__initcall1_start,
	__initcall2_start,
	__initcall3_start,
	__initcall4_start,
	__initcall5_start,
	__initcall6_start,
	__initcall7_start,
	__initcall_end,
};

/*
 * Boot command-line arguments
 */
#define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
#define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT

static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
static const char *panic_later, *panic_param;

extern const struct obs_kernel_param __setup_start[], __setup_end[];

/* Untouched command line saved by arch-specific code. */
char __initdata boot_command_line[COMMAND_LINE_SIZE] __weak;
void *__fdt_pointer;
/* Untouched saved command line (eg. for /proc) */
char *saved_command_line;
/* Command line for parameter parsing */
static char *static_command_line;
/* Command line for per-initcall parameter parsing */
static char *initcall_command_line;
/* notify-chain for early fdt */
static BLOCKING_NOTIFIER_HEAD(of_earlyfdt_chain);

u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };

/* Change NUL term back to "=", to make "param" the whole string. */
static int __init repair_env_string(char *param, char *val,
				    const char *unused, void *arg)
{
	if (val) {
		/* param=val or param="val"? */
		if (val == param+strlen(param)+1)
			val[-1] = '=';
		else if (val == param+strlen(param)+2) {
			val[-2] = '=';
			memmove(val-1, val, strlen(val)+1);
			val--;
		} else
			BUG();
	}
	return 0;
}

/* Anything after -- gets handed straight to init. */
static int __init set_init_arg(char *param, char *val,
			       const char *unused, void *arg)
{
	unsigned int i;

	if (panic_later)
		return 0;

	repair_env_string(param, val, unused, NULL);

	for (i = 0; argv_init[i]; i++) {
		if (i == MAX_INIT_ARGS) {
			panic_later = "init";
			panic_param = param;
			return 0;
		}
	}
	argv_init[i] = param;
	return 0;
}

static int __init obsolete_checksetup(char *line)
{
	const struct obs_kernel_param *p;
	int had_early_param = 0;

	p = __setup_start;
	do {
		int n = strlen(p->str);

		if (parameqn(line, p->str, n)) {
			if (p->early) {
				/* Already done in parse_early_param?
				 * (Needs exact match on param part).
				 * Keep iterating, as we can have early
				 * params and __setups of same names 8( */
				if (line[n] == '\0' || line[n] == '=')
					had_early_param = 1;
			} else if (!p->setup_func) {
				pr_warn("Parameter %s is obsolete, ignored\n",
					p->str);
				return 1;
			} else if (p->setup_func(line + n))
				return 1;
		}
		p++;
	} while (p < __setup_end);

	return had_early_param;
}

/*
 * Unknown boot options get handed to init, unless they look like
 * unused parameters (modprobe will find them in /proc/cmdline).
 */
static int __init unknown_bootoption(char *param, char *val,
				     const char *unused, void *arg)
{
	repair_env_string(param, val, unused, NULL);

	/* Handle obsolete-style parameters */
	if (obsolete_checksetup(param))
		return 0;

	/* Unused module parameter. */
	if (strchr(param, '.') && (!val || strchr(param, '.') < val))
		return 0;

	if (panic_later)
		return 0;

	if (val) {
		/* Environment option */
		unsigned int i;

		for (i = 0; envp_init[i]; i++) {
			if (i == MAX_INIT_ENVS) {
				panic_later = "env";
				panic_param = param;
			}
			if (!strncmp(param, envp_init[i], val - param))
				break;
		}
		envp_init[i] = param;
	} else {
		/* Command line option */
		unsigned int i;

		for (i = 0; argv_init[i]; i++) {
			if (i == MAX_INIT_ARGS) {
				panic_later = "init";
				panic_param = param;
			}
		}
		argv_init[i] = param;
	}
	return 0;
}

/* Check for early params. */
static int __init do_early_param(char *param, char *val,
				 const char *unused, void *arg)
{
	const struct obs_kernel_param *p;

	for (p = __setup_start; p < __setup_end; p++) {
		if ((p->early && parameq(param, p->str)) ||
		    (strcmp(param, "console") == 0 &&
		     strcmp(p->str, "earlycon") == 0)
		) {
			if (p->setup_func(val) != 0)
				pr_warn("Malformed early option '%s'\n", param);
		}
	}
	/* We accept everything at this stage. */
	return 0;
}

void __init parse_early_options(char *cmdline)
{
	parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,
		   do_early_param);
}

/* Arch code calls this early on, or if not, just before other parsing. */
void __init parse_early_param(void)
{
	static int done __initdata;
	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;

	if (done)
		return;

	/* All fallthrough to do_early_param. */
	strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
	parse_early_options(tmp_cmdline);
	done = 1;
}

/*
 * We need to store the untouched command line for future reference.
 * We also need to store the touched command line since the parameter
 * parsing is performed in place, and we should allow a component to
 * store reference of name/value for future reference.
 */
static void __init setup_command_line(char *command_line)
{
	saved_command_line =
		memblock_alloc(strlen(boot_command_line) + 1, 0);
	initcall_command_line =
		memblock_alloc(strlen(boot_command_line) + 1, 0);
	static_command_line = memblock_alloc(strlen(command_line) + 1, 0);
	strcpy(saved_command_line, boot_command_line);
	strcpy(static_command_line, command_line);
}

/* Keep these in sync with initcalls in include/linux/init.h */
static char *initcall_level_names[] __initdata = {
	"early",
	"core",
	"postcore",
	"arch",
	"subsys",
	"fs",
	"device",
	"late",
};

static void __init do_pre_initcalls(void)
{
	initcall_t *fn;

	for (fn = __initcall_start; fn < __initcall0_start; fn++)
		do_one_initcall(*fn);
}

static void __init do_initcall_level(int level)
{
	initcall_t *fn;

	strcpy(initcall_command_line, saved_command_line);
	parse_args(initcall_level_names[level],
			initcall_command_line, __start___param,
			__stop___param - __start___param,
			level, level,
			NULL, &repair_env_string);

	for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
		do_one_initcall(*fn);
}

void __init liblinux_do_initcalls(void)
{
	int level;

	for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++)
		do_initcall_level(level);
}

extern int init_workqueues(void);
extern int kthreadd(void *unused);
extern int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags __attribute__((unused)));
extern void kernel_thread_init(void);
extern void liblinux_cred_init(void);

static inline
void __setup_nr_cpu(void)
{
	static int cpu_nr;
	static unsigned long online_mask;
	unsigned long cpu_map[NR_CPUS];
	int i;

	cpu_nr = liblinux_pal_cpu_config(&online_mask, cpu_map, UL(NR_CPUS),
					    IS_BUILTIN(CONFIG_LIBLINUX_PREEMPT));
	if (cpu_nr < 0)
		panic("get cpu config failed, ret=%d\n", cpu_nr);

	init_cpu_possible(cpu_none_mask);
	init_cpu_online(cpu_none_mask);
	for (i = 0; i < cpu_nr; i++) {
		set_cpu_possible(i, true);
		set_cpu_present(i, (online_mask & (1 << i)) == (1 << i));
#ifndef CONFIG_LIBLINUX_HOTPLUG
		set_cpu_online(i, (online_mask & (1 << i)) == (1 << i));
#endif
		set_cpu_logical_map(i, cpu_map[i] & MPIDR_HWID_BITMASK);
	}
}

static void __init liblinux_vendor_hook_init(void)
{
	inithook_t *fn = NULL;

	for (fn = __vendor_hook_initcall_start; fn < __vendor_hook_initcall_end; fn++)
		(*fn)();
}

void __init proc_caches_init(void)
{
	fs_cachep = kmem_cache_create("fs_cache",
			sizeof(struct fs_struct), 0,
			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
			NULL);
}

void __init setup_arch(char **cmdline_p)
{
	*cmdline_p = boot_command_line;

	if (__fdt_pointer == NULL)
		return;
	if (!early_init_dt_scan(__fdt_pointer))
		panic("invalid fdt\n");

	unflatten_device_tree();
#ifdef CONFIG_HIGHMEM
	kmap_init();
#endif
}

void __init __no_stack_protector liblinux_kernel_init(void)
{
	char *after_dashes;
	char *command_line;

	kernel_thread_init();

	/* clear init thread preempt_count before do-init */
	preempt_count_set(0);
	__setup_nr_cpu();

	local_irq_disable();
	early_boot_irqs_disabled = true;

#ifdef CONFIG_LIBLINUX_HOTPLUG
	boot_cpu_init();
#endif

	page_address_init();
	page_alloc_init();

	/* alloc zero-page */
	empty_zero_page = alloc_page(__GFP_ZERO);
	flush_dcache_page(empty_zero_page);

	setup_arch(&command_line);
	setup_command_line(command_line);
	pr_notice("Kernel command line: %s\n", command_line);
	jump_label_init();
	parse_early_param();
	after_dashes = parse_args("Booting kernel",
				  static_command_line, __start___param,
				  __stop___param - __start___param,
				  -1, -1, NULL, &unknown_bootoption);
	if (!IS_ERR_OR_NULL(after_dashes))
		parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
			   NULL, set_init_arg);
	liblinux_set_param();

	vmalloc_init();

	setup_nr_cpu_ids();
	setup_per_cpu_areas();
#ifdef CONFIG_LIBLINUX_HOTPLUG
	boot_cpu_hotplug_init();
#endif

	/* Set up the the initial canary */
	boot_init_stack_canary();

	kmem_cache_init();
	kmemleak_init();

	liblinux_cred_init();

	early_trace_init();

	radix_tree_init();

	workqueue_init_early();

	rcu_init();

	trace_init();

	liblinux_vendor_hook_init();

	early_irq_init();

	init_timers();
	hrtimers_init();
	softirq_init();
	timekeeping_init();
	random_init(command_line);

	early_boot_irqs_disabled = false;
	local_irq_enable();

	(void)kernel_thread(kthreadd, NULL, 0);

	workqueue_init();
	kmem_cache_init_late();
	console_init();

	vfs_caches_init_early();
	wait_bit_init(); /* sched_init() */

	sched_clock_init();

	/* finish all async __init code before change state to RUNNING */
	async_synchronize_full();
	system_state = SYSTEM_RUNNING;

	gfp_allowed_mask = __GFP_BITS_MASK;

	rcu_end_inkernel_boot();

	proc_caches_init();
	buffer_init();
	key_init();
	vfs_caches_init();
	pagecache_init();
	seq_file_init();
	proc_root_init();
	init_irq_proc();
	do_pre_initcalls();
	if (liblinux_bootfdt_enable())
		of_clk_init(NULL);

#ifdef CONFIG_LIBLINUX_HOTPLUG
	smp_init();
#endif
	driver_init();
}

asmlinkage __visible void __init start_kernel(void)
{
	liblinux_kernel_init();
	liblinux_do_initcalls();
}

unsigned long loops_per_jiffy = (1<<12);
EXPORT_SYMBOL(loops_per_jiffy);

struct pglist_data __refdata contig_page_data;
EXPORT_SYMBOL(contig_page_data);

/* of_fdt required arch memblock APIs, overwrite weak symbol in "driver/of/fdt.c" */
int __init early_init_dt_reserve_memory_arch(phys_addr_t base,
				      phys_addr_t size, bool nomap)
{
	/*
	 * DO NOTHING - The reserved-memory detected by fdt is useless for liblinux
	 */
	return 0;
}

extern int liblinux_kasan_populate_shadow(const void *addr, size_t len);

void __init early_init_dt_add_memory_arch(u64 base, u64 size)
{
#ifdef CONFIG_LIBLINUX_NATIVE_KASAN
	liblinux_kasan_populate_shadow(kasan_mem_to_shadow((void *)(unsigned long)base),
				       size >> KASAN_SHADOW_SCALE_SHIFT);
#endif
}

void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
{
	return memblock_alloc(size, align);
}

int liblinux_bootfdt_enable(void)
{
	return __fdt_pointer != NULL;
}

int __init liblinux_register_irqchip_table(const struct of_device_id *table)
{
	of_irq_init(table);
	return 0;
}
EXPORT_SYMBOL(liblinux_register_irqchip_table);

#ifdef CONFIG_OF_EARLY_FLATTREE
int of_register_earlyfdt_notifier(struct notifier_block *nb)
{
	if (of_have_populated_dt())
		return -EINVAL;

	return blocking_notifier_chain_register(&of_earlyfdt_chain, nb);
}
EXPORT_SYMBOL_GPL(of_register_earlyfdt_notifier);

int __init liblinux_setup_fdt(void *dt_virt)
{
	struct device_node *np;
	int ret;

	if (of_have_populated_dt())
		return -EBUSY;

	if (!dt_virt || !early_init_dt_scan(dt_virt)) {
		pr_crit("liblinux: invalid device tree blob.\n");
		while (true)
			cpu_relax();
	}

	/* include __unflatten_dt() and of_alis_scan() */
	unflatten_device_tree();

	/* re-do sysfs works in of_core_init() */
	mutex_lock(&of_mutex);
	for_each_of_allnodes(np)
		__of_attach_node_sysfs(np);
	mutex_unlock(&of_mutex);
	proc_symlink("device-tree", NULL, "/sys/firmware/devicetree/base");

	/* call earlyfdt notifier */
	ret = blocking_notifier_call_chain(&of_earlyfdt_chain, 0, NULL);
	if (ret < 0)
		return ret;

	/* of misc init */
	of_clk_init(NULL);
	early_init_fdt_scan_reserved_mem();

	return 0;
}
EXPORT_SYMBOL_GPL(liblinux_setup_fdt);
#else
int of_register_earlyfdt_notifier(struct notifier_block *nb)
{
	return 0;
}
EXPORT_SYMBOL_GPL(of_register_earlyfdt_notifier);

int liblinux_setup_fdt(void *dt_virt)
{
	return 0;
}
EXPORT_SYMBOL_GPL(liblinux_setup_fdt);
#endif

/* arch/arm64/kernel/smp.c */
void __cpu_die(unsigned int cpu)
{
	pr_notice("CPU%u: shutdown\n", cpu);
}
int __weak __cpu_disable(void)
{
	pr_notice("CPU%u: disabled\n", smp_processor_id());
	return 0;
}

int __weak __cpu_up(unsigned int cpu, struct task_struct *idle)
{
	pr_notice("CPU%u: up\n", cpu);
	return 0;
}
DEFINE_PER_CPU(bool, cpu_dead_idle);

/* a fake stop_machine */
#include <linux/stop_machine.h>
int __weak stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
{
	unsigned long flags;
	int ret;

	local_irq_save(flags);
	ret = fn(data);
	local_irq_restore(flags);
	return ret;
}
EXPORT_SYMBOL_GPL(stop_machine);

/* arch interrupt support ("arch/{arm64,arm}/kernel/irq.c */
unsigned long irq_err_count;
int arch_show_interrupts(struct seq_file *p, int prec)
{
	seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
	return 0;
}

bool static_key_initialized __read_mostly;
EXPORT_SYMBOL_GPL(static_key_initialized);
